import requests
import pandas as pd
import json
from time import sleep

headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.69'
    }
url = 'http://price.mofcom.gov.cn/datamofcom/front/price/pricequotation/priceQuery'

def main():
    farm_produce()

def farm_produce():
    info = []
    for i in range(1,1867):
        params = {
            'startTime': None,
            'endTime': None,
            'pageNumber': i,
            'pageSize': 20,
            'pro_name': None,
            'pro_trade': None,
            'pro_region': '国内',
            'pro_type': '粮农土畜',
        }
        response = requests.post(url, headers=headers, params=params)
        data = json.loads(response.text)
        data_list = data.get('rows', [])
        info += [[i['seqno'], i['prod_id'], i['prod_name'], i['prod_spec'], i['price'], i['unit'], i['yyyy'], i['mm'],
                 i['dd']] for i in data_list]
        print(f'第{i}页爬取完成')
        sleep(2)
    df = pd.DataFrame(info, columns=['序号', '产品代码', '产品名称', '产品规格', '价格', '单位', '年份', '月份', '日期'])
    df.to_csv('../static/data/farm_produce.csv', index=False)


if __name__ == '__main__':
    main()